package com.zouxian.news;
//SKY-home-新闻
//https://news.sky.com/
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.util.List;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;

import com.zouxian.pipeline.MyFilePipeline;
import com.zouxian.pojo.News;
import com.zouxian.util.ListToString;
import com.zouxian.util.StringUtils;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.selenium.SeleniumDownloader;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

public class MyProcessor3  implements PageProcessor{
	//配置
	private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
	private static int count=0;
	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}

	public void process(Page page) {
		//新建一个新闻对象
		News news=new News();
		String resource="https://news.sky.com";
		//https://news.sky.com/story/grisly-industry-bodies-share-fears-about-governments-no-deal-brexit-plans-11507839
		//https://news.sky.com/story/live-inquest-into-death-of-girl-who-ate-pret-baguette-11508049
		//https://news.sky.com/story/man-26-arrested-after-missing-woman-found-dead-in-west-london-home-11507904
		//https://news.sky.com/story/uk-to-call-for-third-of-worlds-oceans-to-be-protected-by-2030-11507876
		if(page.getUrl().regex(resource).match()){
			List<String> all = page.getHtml().xpath("//*[@class=\"sdc-news-header__nav-group\"]/li/a/@href").all();
			if(!all.toString().contains("http")){
				List<String> url = StringUtils.addURL(all, resource);
				page.addTargetRequests(url);
			}else{
				page.addTargetRequests(all);
			}
		}
		Boolean flag=page.getUrl().regex("https://news.sky.com/story/[A-Z a-z 0-9 -]+").match();
		if(!flag){
			//获取所有满足条件的url
			page.addTargetRequests(page.getHtml().links().regex("https://news.sky.com/story/[A-Z a-z 0-9 -]+").all());
			
		}
		//System.err.println(page.getUrl());
		count++;
		//获取url
		String Url=page.getUrl().get();
		news.setUrl(Url);
		//获取标题..
		//String title=page.getHtml().xpath("//*[@id=\"article\"]/div[2]/div/div[1]/header/div[3]/div[1]/div/h1/font").get();
		String title=page.getHtml().$("h1.sdc-news-article-header__headline","allText").all().toString();
		
		if(title==null){
			System.out.println("该链接没有标题，不符合要求！");
			page.setSkip(true);
			
		}
		news.setTitle(title);
		//获取关键字
		//String articalKeyWords=page.getHtml().$("div.submeta__section-labels ul li a","allText").all().toString();
		//news.setArticalKeyWords(articalKeyWords);
		//获取日期.
		String GetTime=page.getHtml().$("span.sdc-news-article-header__last-updated__time","allText").all().toString();
		news.setGetTime(GetTime);
		//获取正文.
		String Txt=page.getHtml().$("div.site-main", "allText").all().toString();
		news.setTxt(Txt);
		//获取摘要 .
		String Summary=page.getHtml().$("p.sdc-news-article-header__standfirst", "allText").all().toString();
		news.setSummary(Summary);
		//获取文章类型.
		//String TxtType=page.getHtml().$("div[class='article-type publication-font']:first-child a:first-child","allText").all().toString();
		//news.setTxtType(TxtType);
		//获取作者.
		String author=page.getHtml().$("p.sdc-news-story-article__byline", "allText").all().toString();
		news.setAuthor(author);
		page.putField("title", news.getTitle());
		page.putField("TitleCh", news.getTitleCh());
		page.putField("Title_Score", news.getTitle_Score());
		page.putField("articalKeyWords", news.getArticalKeyWords());
		page.putField("Txt", news.getTxt());
		page.putField("TxtCh", news.getTxtCh());
		page.putField("Txt_Score", news.getTxt_Score());
		page.putField("Url", news.getUrl());
		page.putField("Summary", news.getSummary());
		page.putField("SummaryCh", news.getSummaryCh());
		page.putField("TxtType", news.getTxtType());
		page.putField("GetTime", news.getGetTime());
		page.putField("opicType", news.getOpicType());
		page.putField("Source", news.getSource());
		page.putField("isdown", news.getIsdown());
		page.putField("relateNewsNumber", news.getRelateNewsNumber());
		page.putField("KeyWords", news.getKeyWords());
		page.putField("schoolName", news.getSchoolName());
		page.putField("country", news.getCountry());
		page.putField("Polar", news.getPolar());
		page.putField("author", news.getAuthor());
		page.putField("language", news.getLanguage());
		page.putField("htmlCode", news.getHtmlCode());
		page.putField("semanticValue", news.getSemanticValue());
		page.putField("mediaIntroduction", news.getMediaIntroduction());
		page.putField("zuanfaNum", news.getZuanfaNum());
		page.putField("pinlunNum", news.getPinlunNum());
		page.putField("shouchangNum", news.getSchoolName());
		page.putField("txtSeg", news.getTxtSeg());
	}
	
	//.setDownloader(new SeleniumDownloader("C:\\Users\\zx\\Downloads\\chromedriver.exe"))
	public static void main(String[] args) {
		long startTime, endTime;
		System.out.println("开始爬取...");
		startTime = System.currentTimeMillis();
		Spider.create(new MyProcessor3()).addUrl("https://news.sky.com").addPipeline(new ConsolePipeline()).thread(5).run();
		endTime = System.currentTimeMillis();
		System.out.println("爬取结束，耗时约" + ((endTime - startTime) / 1000)
				+ "秒，抓取了" + count + "条记录");
		
	}

}
